#coding=utf-8
import urllib,urllib2
from BeautifulSoup import BeautifulSoup
import socket
socket.setdefaulttimeout(10)

import datetime
today = datetime.date.today()
ey = today.year
em = today.month

def get_soup(code, sy,sm,sd, ey,em,ed, coding='gb18030'):
    tj = unicode('提交', 'u8').encode('gbk')
 
    ymd = [sy,sm,sd,ey,em,ed]
    ymd = [str(i)for i in ymd]
    for i in range(len(ymd)):
        if len(ymd[i]) == 1:
             ymd[i] = '0' + ymd[i]
    #print ymd
             
    ps = [('code', str(code)), ('submit', tj),
          ('start_year', ymd[0]), ('start_month', ymd[1]), ('start_day', ymd[2]),
          ('end_year', ymd[3]), ('end_month', ymd[4]), ('end_day', ymd[5]),
          ('ss', 'd')]
    url = 'http://stock.business.sohu.com/q/hp.php'
    
    data = urllib.urlencode(ps)
    req = urllib2.Request(url)
    while True:
        try:
            f = urllib2.urlopen(req, data)
            data = f.read()
            break
        except (urllib2.URLError,socket.timeout,IOError), e:
            pass
    
    htmltext = data.decode(coding, 'ignore')
    s = BeautifulSoup(htmltext)
    
    return s
            
def price_from_soup(soup):
    
    def remove_comma(data):
        return ''.join( data.split(',') )
    
    def parse_row(row):
        r = []
        tbs = row.findAll('td')
         
        for tb in tbs:
            s = tb.string
            if not s: s = tb.b.string
            r.append(s)
            
        r = [i.encode('u8') for i in r]
        
        dt = r[0]
        dt = '-'.join( [dt[:4], dt[4:6], dt[6:8]] )
        r[0] = dt
        
        r = [remove_comma(i) for i in r]
        return r
    
    data = []
    rq = unicode('日期', 'u8')
    rqtext = soup.find(text=rq)
    if not rqtext:return None
    tb = rqtext.parent.parent.parent.parent
    trs = tb.findAll('tr')[1:]
    for tr in trs:
        data.append( parse_row(tr) )
        
    return data

def yms(sy, sm, ey, em):
    r = []
    if sy == ey:
        for m in range(sm, em+1):
            r.append([sy, m])
        return r
    
    for m in range(sm, 13):
        r.append([sy, m])
    
    for y in range(sy+1, ey):
        for i in range(1,13):
            r.append([y, i])
    
    for m in range(1, em+1):
        r.append([ey, m])
        
    return r

def ymds(yms):
    r = []
    for ym in yms:
        y, m = ym
        r.append([y,m,1,y,m,31])
    return r

def get_ymds(sy,sm,ey,em):
    d = yms(sy,sm,ey,em)
    r = ymds(d)
    return r

def get_price_since(code, exdate):
    p = []
    sy,sm,sd = map(int, exdate.split('-'))
    for ymd in get_ymds(sy,sm,ey,em):
        print ymd
        soup = get_soup(code, *ymd)
        data = price_from_soup(soup)
        if data:p.extend(data)
    return p

if __name__ == '__main__':
    from stsettings import datadir
    from stutils import data2csv
    
    cns = {'1A0001':'上证指数', '399001':'深证成指'}
    index__head = ["日期", "开盘价", "最高价", "收盘价", "最低价","交易量(手)", "交易金额(万元)"]
    for k in cns.keys():
        fn = '%s/%s.csv'%(datadir, cns[k])
        print fn
        
        data = get_price_since(k, '1990-12-01')
        data2csv(data, index_head, fn)    
        
